{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intro\n",
    "\n",
    "This is inspired by \n",
    "Inproceedings (magdon1998neural) Magdon-Ismail, M. & Atiya, A. Neural networks for density estimation NIPS, 1998, 522-528\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using StatsBase\n",
    "using Distributions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using TensorFlow\n",
    "using MLDataUtils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using DensityEstimationML\n",
    "using Plots\n",
    "\n",
    "using MacroTools\n",
    "macro plot(ex)\n",
    "    @capture(ex, (x_, [ys__], tail__ )) \n",
    "    labels = repr.(ys)\n",
    "    ys_expr = Expr(:hvect, ys...)\n",
    "    labels_expr = Expr(:vect, repr.(ys)...)\n",
    "    Expr(:call, :plot, x, ys_expr, tail..., Expr(:kw, :labels, labels_expr))\n",
    "end\n",
    "\n",
    "@plot (X, [Z, logistic.(Z)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "immutable NeuralDensityEstimator\n",
    "    sess::Session\n",
    "    \n",
    "    #Network nodes\n",
    "    optimizer::Tensor\n",
    "    t::Tensor\n",
    "    pdf::Tensor\n",
    "    cdf::Tensor\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "abstract type NdeModel; end\n",
    "immutable NdeNaive <: NdeModel; end\n",
    "immutable NdeAsymptoticForced <: NdeModel; end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "leaky_relu6(z) = 0.001z + nn.relu6(z)\n",
    "\n",
    "NeuralDensityEstimator(prob_layer_sizes) = NeuralDensityEstimator(prob_layer_sizes, NdeNaive())\n",
    "\n",
    "function NeuralDensityEstimator(prob_layer_sizes, model::NdeNaive)\n",
    "    sess = Session(Graph())\n",
    "    @tf begin\n",
    "        t1 = placeholder(Float32, shape=[1, -1])\n",
    "        t2 = placeholder(Float32, shape=[1, -1])\n",
    "        countprob_between = placeholder(Float32, shape=[-1])\n",
    "        \n",
    "        zp1 = [t1]\n",
    "        zp2 = [t2]\n",
    "        layer_sizes= [1; prob_layer_sizes; 1]\n",
    "        act_funs = Vector{Function}(length(layer_sizes)-1)\n",
    "        act_funs[:] = nn.relu6\n",
    "        act_funs[end] = nn.sigmoid\n",
    "        for ii in 2:length(layer_sizes)\n",
    "            below_size = layer_sizes[ii-1]\n",
    "            above_size = layer_sizes[ii]\n",
    "            \n",
    "            act_fun = act_funs[ii-1]\n",
    "            \n",
    "            Wii = get_variable(\"W_$ii\", [above_size, below_size], Float32)\n",
    "            bii = get_variable(\"b_$ii\", [above_size, 1], Float32)\n",
    "            \n",
    "            push!(zp1, act_fun(Wii*zp1[end] .+ bii))\n",
    "            push!(zp2, act_fun(Wii*zp2[end] .+ bii))\n",
    "        end\n",
    "        \n",
    "        cdf1 = zp1[end]\n",
    "        cdf2 = zp2[end]\n",
    "        #Assumes t2>t1\n",
    "        cdf_between = cdf2 - cdf1\n",
    "                \n",
    "        \n",
    "        losses = 0.5.*(cdf_between .- countprob_between).^2\n",
    "        \n",
    "        loss=reduce_mean(losses; axis=2)\n",
    "        optimizer = train.minimize(train.AdamOptimizer(), loss)\n",
    "        \n",
    "        pdf1 = gradients(cdf1, t1)\n",
    "    end\n",
    "    \n",
    "    run(sess, global_variables_initializer())\n",
    "    \n",
    "    NeuralDensityEstimator(sess, optimizer, t1, pdf1, cdf1)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tanh.(-1000:1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot(x->logistic(x*logistic(10x-2)), xlims=(-10, 10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function NeuralDensityEstimator(prob_layer_sizes, model::NdeAsymptoticForced)\n",
    "    sess = Session(Graph())\n",
    "    @tf begin\n",
    "        t1 = placeholder(Float32, shape=[1, -1])\n",
    "        t2 = placeholder(Float32, shape=[1, -1])\n",
    "        countprob_between = placeholder(Float32, shape=[-1])\n",
    "        \n",
    "        zp1 = [t1]\n",
    "        zp2 = [t2]\n",
    "        layer_sizes= [1; prob_layer_sizes; 1]\n",
    "        act_funs = Vector{Function}(length(layer_sizes)-1)\n",
    "        act_funs[:] = nn.relu6\n",
    "        act_funs[end] = TensorFlow.identity\n",
    "        for ii in 2:length(layer_sizes)\n",
    "            below_size = layer_sizes[ii-1]\n",
    "            above_size = layer_sizes[ii]\n",
    "            \n",
    "            act_fun = act_funs[ii-1]\n",
    "            \n",
    "            Wii = get_variable(\"W_$ii\", [above_size, below_size], Float32)\n",
    "            bii = get_variable(\"b_$ii\", [above_size, 1], Float32)\n",
    "            \n",
    "            push!(zp1, act_fun(Wii*zp1[end] .+ bii))\n",
    "            push!(zp2, act_fun(Wii*zp2[end] .+ bii))\n",
    "        end\n",
    "        \n",
    "        #forces assymtoptess    \n",
    "        s_shift = get_variable([1], Float32) #support translate\n",
    "        s_scale = get_variable([1], Float32) #support scale\n",
    "        cft_fun(z, t) = nn.sigmoid(2(s_scale*t + s_shift)) + tanh(s_scale*t + s_shift)^2 * z\n",
    "                    \n",
    "        cdf1 = cft_fun(zp1[end], t1)\n",
    "        cdf2 = cft_fun(zp2[end], t2)\n",
    "        #Assumes t2>t1\n",
    "        cdf_between = cdf2 - cdf1\n",
    "                \n",
    "        \n",
    "        losses = 0.5.*(cdf_between .- countprob_between).^2\n",
    "        \n",
    "        loss=reduce_mean(losses; axis=2)\n",
    "        optimizer = train.minimize(train.AdamOptimizer(), loss)\n",
    "        \n",
    "        pdf1 = gradients(cdf1, t1)\n",
    "    end\n",
    "    \n",
    "    run(sess, global_variables_initializer())\n",
    "    \n",
    "    NeuralDensityEstimator(sess, optimizer, t1, pdf1, cdf1)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "using StatsFuns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[33mWARNING: \u001b[39m\u001b[22m\u001b[33m.* is no longer a function object; use broadcast(*, ...) instead\u001b[39m\n",
      "Stacktrace:\n",
      " [1] \u001b[1mdepwarn\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::Symbol\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./deprecated.jl:70\u001b[22m\u001b[22m\n",
      " [2] \u001b[1m(::Base.##716#717)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}, ::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./deprecated.jl:346\u001b[22m\u001b[22m\n",
      " [3] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m\n",
      " [4] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Module, ::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/Compat/src/Compat.jl:577\u001b[22m\u001b[22m\n",
      " [5] \u001b[1mexecute_request\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::ZMQ.Socket, ::IJulia.Msg\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/IJulia/src/execute_request.jl:154\u001b[22m\u001b[22m\n",
      " [6] \u001b[1meventloop\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::ZMQ.Socket\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/IJulia/src/eventloop.jl:8\u001b[22m\u001b[22m\n",
      " [7] \u001b[1m(::IJulia.##14#17)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./task.jl:335\u001b[22m\u001b[22m\n",
      "while loading In[75], in expression starting on line 3\n",
      "\u001b[1m\u001b[33mWARNING: \u001b[39m\u001b[22m\u001b[33m.* is no longer a function object; use broadcast(*, ...) instead\u001b[39m\n",
      "Stacktrace:\n",
      " [1] \u001b[1mdepwarn\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::Symbol\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./deprecated.jl:70\u001b[22m\u001b[22m\n",
      " [2] \u001b[1m(::Base.##716#717)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Array{Float64,1}, ::Array{Float64,1}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./deprecated.jl:346\u001b[22m\u001b[22m\n",
      " [3] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m\n",
      " [4] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Module, ::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/Compat/src/Compat.jl:577\u001b[22m\u001b[22m\n",
      " [5] \u001b[1mexecute_request\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::ZMQ.Socket, ::IJulia.Msg\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/IJulia/src/execute_request.jl:154\u001b[22m\u001b[22m\n",
      " [6] \u001b[1meventloop\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::ZMQ.Socket\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/IJulia/src/eventloop.jl:8\u001b[22m\u001b[22m\n",
      " [7] \u001b[1m(::IJulia.##14#17)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./task.jl:335\u001b[22m\u001b[22m\n",
      "while loading In[75], in expression starting on line 3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 600 400\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip00\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"600\" height=\"400\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "0,400 600,400 600,0 0,0 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip01\">\n",
       "    <rect x=\"120\" y=\"0\" width=\"421\" height=\"400\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "39.3701,376.378 592.126,376.378 592.126,7.87402 39.3701,7.87402 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip02\">\n",
       "    <rect x=\"39\" y=\"7\" width=\"554\" height=\"370\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  39.3701,370.85 39.3701,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  177.559,370.85 177.559,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  315.748,370.85 315.748,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  453.937,370.85 453.937,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  592.126,370.85 592.126,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  47.6614,319.436 583.835,319.436 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  47.6614,241.363 583.835,241.363 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  47.6614,163.29 583.835,163.29 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  47.6614,85.2168 583.835,85.2168 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 592.126,376.378 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 39.3701,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  177.559,376.378 177.559,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  315.748,376.378 315.748,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  453.937,376.378 453.937,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  592.126,376.378 592.126,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 39.3701,7.87402 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,319.436 47.6614,319.436 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,241.363 47.6614,241.363 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,163.29 47.6614,163.29 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,85.2168 47.6614,85.2168 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 39.3701, 390.178)\" x=\"39.3701\" y=\"390.178\">-2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 177.559, 390.178)\" x=\"177.559\" y=\"390.178\">-1</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 315.748, 390.178)\" x=\"315.748\" y=\"390.178\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 453.937, 390.178)\" x=\"453.937\" y=\"390.178\">1</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 592.126, 390.178)\" x=\"592.126\" y=\"390.178\">2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 33.3701, 323.936)\" x=\"33.3701\" y=\"323.936\">-1</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 33.3701, 245.863)\" x=\"33.3701\" y=\"245.863\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 33.3701, 167.79)\" x=\"33.3701\" y=\"167.79\">1</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 33.3701, 89.7168)\" x=\"33.3701\" y=\"89.7168\">2</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#009af9; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 40.752,375.958 42.1339,375.537 43.5157,375.113 44.8976,374.687 46.2795,374.259 47.6614,373.829 49.0433,373.397 50.4252,372.963 51.8071,372.526 \n",
       "  53.189,372.087 54.5709,371.646 55.9528,371.202 57.3346,370.756 58.7165,370.308 60.0984,369.857 61.4803,369.403 62.8622,368.947 64.2441,368.488 65.626,368.027 \n",
       "  67.0079,367.563 68.3898,367.096 69.7717,366.627 71.1535,366.154 72.5354,365.679 73.9173,365.201 75.2992,364.72 76.6811,364.236 78.063,363.749 79.4449,363.259 \n",
       "  80.8268,362.765 82.2087,362.269 83.5906,361.769 84.9724,361.266 86.3543,360.76 87.7362,360.25 89.1181,359.737 90.5,359.221 91.8819,358.701 93.2638,358.177 \n",
       "  94.6457,357.65 96.0276,357.119 97.4094,356.585 98.7913,356.047 100.173,355.505 101.555,354.959 102.937,354.409 104.319,353.856 105.701,353.298 107.083,352.737 \n",
       "  108.465,352.171 109.846,351.601 111.228,351.027 112.61,350.449 113.992,349.867 115.374,349.28 116.756,348.689 118.138,348.094 119.52,347.494 120.902,346.89 \n",
       "  122.283,346.281 123.665,345.667 125.047,345.049 126.429,344.427 127.811,343.799 129.193,343.167 130.575,342.53 131.957,341.889 133.339,341.242 134.72,340.591 \n",
       "  136.102,339.935 137.484,339.274 138.866,338.608 140.248,337.937 141.63,337.26 143.012,336.579 144.394,335.893 145.776,335.202 147.157,334.505 148.539,333.804 \n",
       "  149.921,333.097 151.303,332.385 152.685,331.668 154.067,330.945 155.449,330.218 156.831,329.485 158.213,328.747 159.594,328.004 160.976,327.256 162.358,326.502 \n",
       "  163.74,325.744 165.122,324.98 166.504,324.211 167.886,323.437 169.268,322.657 170.65,321.873 172.031,321.084 173.413,320.289 174.795,319.49 176.177,318.685 \n",
       "  177.559,317.876 178.941,317.062 180.323,316.243 181.705,315.419 183.087,314.591 184.469,313.758 185.85,312.92 187.232,312.079 188.614,311.232 189.996,310.382 \n",
       "  191.378,309.527 192.76,308.668 194.142,307.805 195.524,306.938 196.906,306.067 198.287,305.193 199.669,304.315 201.051,303.434 202.433,302.55 203.815,301.662 \n",
       "  205.197,300.772 206.579,299.878 207.961,298.982 209.343,298.084 210.724,297.183 212.106,296.28 213.488,295.375 214.87,294.469 216.252,293.56 217.634,292.651 \n",
       "  219.016,291.74 220.398,290.828 221.78,289.916 223.161,289.003 224.543,288.09 225.925,287.176 227.307,286.263 228.689,285.351 230.071,284.439 231.453,283.528 \n",
       "  232.835,282.618 234.217,281.71 235.598,280.803 236.98,279.899 238.362,278.997 239.744,278.098 241.126,277.201 242.508,276.308 243.89,275.418 245.272,274.532 \n",
       "  246.654,273.65 248.035,272.772 249.417,271.9 250.799,271.032 252.181,270.169 253.563,269.313 254.945,268.462 256.327,267.617 257.709,266.78 259.091,265.949 \n",
       "  260.472,265.125 261.854,264.309 263.236,263.501 264.618,262.701 266,261.909 267.382,261.127 268.764,260.353 270.146,259.589 271.528,258.835 272.909,258.091 \n",
       "  274.291,257.357 275.673,256.634 277.055,255.921 278.437,255.22 279.819,254.531 281.201,253.853 282.583,253.187 283.965,252.534 285.346,251.893 286.728,251.265 \n",
       "  288.11,250.65 289.492,250.048 290.874,249.46 292.256,248.885 293.638,248.324 295.02,247.778 296.402,247.245 297.783,246.727 299.165,246.224 300.547,245.735 \n",
       "  301.929,245.261 303.311,244.802 304.693,244.359 306.075,243.93 307.457,243.517 308.839,243.119 310.22,242.737 311.602,242.37 312.984,242.018 314.366,241.683 \n",
       "  315.748,241.363 317.13,241.058 318.512,240.769 319.894,240.496 321.276,240.238 322.657,239.996 324.039,239.769 325.421,239.558 326.803,239.362 328.185,239.181 \n",
       "  329.567,239.015 330.949,238.865 332.331,238.729 333.713,238.608 335.094,238.501 336.476,238.409 337.858,238.331 339.24,238.267 340.622,238.217 342.004,238.181 \n",
       "  343.386,238.158 344.768,238.149 346.15,238.152 347.531,238.169 348.913,238.197 350.295,238.239 351.677,238.292 353.059,238.357 354.441,238.433 355.823,238.521 \n",
       "  357.205,238.619 358.587,238.729 359.969,238.848 361.35,238.978 362.732,239.117 364.114,239.266 365.496,239.424 366.878,239.591 368.26,239.767 369.642,239.95 \n",
       "  371.024,240.142 372.406,240.341 373.787,240.547 375.169,240.76 376.551,240.98 377.933,241.206 379.315,241.439 380.697,241.676 382.079,241.92 383.461,242.168 \n",
       "  384.843,242.421 386.224,242.678 387.606,242.94 388.988,243.205 390.37,243.474 391.752,243.745 393.134,244.02 394.516,244.298 395.898,244.578 397.28,244.859 \n",
       "  398.661,245.143 400.043,245.428 401.425,245.715 402.807,246.002 404.189,246.29 405.571,246.578 406.953,246.867 408.335,247.156 409.717,247.444 411.098,247.732 \n",
       "  412.48,248.019 413.862,248.305 415.244,248.59 416.626,248.874 418.008,249.156 419.39,249.436 420.772,249.715 422.154,249.991 423.535,250.265 424.917,250.536 \n",
       "  426.299,250.805 427.681,251.071 429.063,251.334 430.445,251.594 431.827,251.85 433.209,252.104 434.591,252.353 435.972,252.599 437.354,252.842 438.736,253.08 \n",
       "  440.118,253.314 441.5,253.544 442.882,253.771 444.264,253.992 445.646,254.21 447.028,254.423 448.409,254.631 449.791,254.835 451.173,255.034 452.555,255.228 \n",
       "  453.937,255.418 455.319,255.602 456.701,255.782 458.083,255.957 459.465,256.127 460.846,256.292 462.228,256.452 463.61,256.606 464.992,256.756 466.374,256.9 \n",
       "  467.756,257.04 469.138,257.174 470.52,257.303 471.902,257.426 473.283,257.545 474.665,257.658 476.047,257.766 477.429,257.869 478.811,257.967 480.193,258.059 \n",
       "  481.575,258.147 482.957,258.229 484.339,258.306 485.72,258.378 487.102,258.445 488.484,258.506 489.866,258.563 491.248,258.614 492.63,258.661 494.012,258.703 \n",
       "  495.394,258.739 496.776,258.771 498.157,258.797 499.539,258.819 500.921,258.836 502.303,258.849 503.685,258.856 505.067,258.859 506.449,258.857 507.831,258.85 \n",
       "  509.213,258.839 510.594,258.823 511.976,258.803 513.358,258.778 514.74,258.749 516.122,258.716 517.504,258.678 518.886,258.635 520.268,258.589 521.65,258.538 \n",
       "  523.031,258.484 524.413,258.425 525.795,258.362 527.177,258.295 528.559,258.224 529.941,258.149 531.323,258.07 532.705,257.987 534.087,257.901 535.469,257.811 \n",
       "  536.85,257.717 538.232,257.619 539.614,257.518 540.996,257.414 542.378,257.306 543.76,257.194 545.142,257.079 546.524,256.961 547.906,256.839 549.287,256.714 \n",
       "  550.669,256.586 552.051,256.455 553.433,256.32 554.815,256.183 556.197,256.042 557.579,255.899 558.961,255.752 560.343,255.603 561.724,255.451 563.106,255.296 \n",
       "  564.488,255.138 565.87,254.977 567.252,254.814 568.634,254.648 570.016,254.48 571.398,254.309 572.78,254.135 574.161,253.959 575.543,253.781 576.925,253.6 \n",
       "  578.307,253.416 579.689,253.231 581.071,253.043 582.453,252.853 583.835,252.66 585.217,252.466 586.598,252.269 587.98,252.07 589.362,251.869 590.744,251.666 \n",
       "  592.126,251.461 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#e26f46; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,229.599 40.752,229.546 42.1339,229.491 43.5157,229.437 44.8976,229.381 46.2795,229.326 47.6614,229.269 49.0433,229.213 50.4252,229.156 51.8071,229.098 \n",
       "  53.189,229.04 54.5709,228.981 55.9528,228.922 57.3346,228.862 58.7165,228.801 60.0984,228.74 61.4803,228.679 62.8622,228.616 64.2441,228.554 65.626,228.49 \n",
       "  67.0079,228.426 68.3898,228.362 69.7717,228.296 71.1535,228.23 72.5354,228.164 73.9173,228.096 75.2992,228.028 76.6811,227.96 78.063,227.89 79.4449,227.82 \n",
       "  80.8268,227.749 82.2087,227.678 83.5906,227.605 84.9724,227.532 86.3543,227.458 87.7362,227.383 89.1181,227.308 90.5,227.231 91.8819,227.154 93.2638,227.076 \n",
       "  94.6457,226.997 96.0276,226.917 97.4094,226.837 98.7913,226.755 100.173,226.672 101.555,226.589 102.937,226.504 104.319,226.419 105.701,226.332 107.083,226.245 \n",
       "  108.465,226.156 109.846,226.067 111.228,225.976 112.61,225.884 113.992,225.792 115.374,225.698 116.756,225.603 118.138,225.507 119.52,225.409 120.902,225.311 \n",
       "  122.283,225.211 123.665,225.11 125.047,225.008 126.429,224.905 127.811,224.8 129.193,224.694 130.575,224.587 131.957,224.479 133.339,224.369 134.72,224.258 \n",
       "  136.102,224.145 137.484,224.031 138.866,223.916 140.248,223.799 141.63,223.681 143.012,223.561 144.394,223.44 145.776,223.318 147.157,223.194 148.539,223.068 \n",
       "  149.921,222.941 151.303,222.812 152.685,222.682 154.067,222.55 155.449,222.417 156.831,222.282 158.213,222.145 159.594,222.007 160.976,221.867 162.358,221.726 \n",
       "  163.74,221.582 165.122,221.438 166.504,221.291 167.886,221.143 169.268,220.993 170.65,220.841 172.031,220.688 173.413,220.533 174.795,220.376 176.177,220.218 \n",
       "  177.559,220.058 178.941,219.896 180.323,219.732 181.705,219.567 183.087,219.4 184.469,219.231 185.85,219.06 187.232,218.888 188.614,218.714 189.996,218.539 \n",
       "  191.378,218.361 192.76,218.182 194.142,218.002 195.524,217.82 196.906,217.636 198.287,217.451 199.669,217.264 201.051,217.075 202.433,216.885 203.815,216.694 \n",
       "  205.197,216.501 206.579,216.307 207.961,216.111 209.343,215.914 210.724,215.716 212.106,215.516 213.488,215.315 214.87,215.113 216.252,214.91 217.634,214.706 \n",
       "  219.016,214.501 220.398,214.295 221.78,214.088 223.161,213.88 224.543,213.671 225.925,213.462 227.307,213.252 228.689,213.041 230.071,212.83 231.453,212.619 \n",
       "  232.835,212.407 234.217,212.194 235.598,211.982 236.98,211.769 238.362,211.557 239.744,211.344 241.126,211.132 242.508,210.919 243.89,210.708 245.272,210.496 \n",
       "  246.654,210.285 248.035,210.074 249.417,209.865 250.799,209.655 252.181,209.447 253.563,209.24 254.945,209.034 256.327,208.829 257.709,208.625 259.091,208.422 \n",
       "  260.472,208.221 261.854,208.022 263.236,207.824 264.618,207.628 266,207.433 267.382,207.241 268.764,207.051 270.146,206.862 271.528,206.676 272.909,206.492 \n",
       "  274.291,206.311 275.673,206.132 277.055,205.955 278.437,205.782 279.819,205.61 281.201,205.442 282.583,205.277 283.965,205.114 285.346,204.955 286.728,204.798 \n",
       "  288.11,204.645 289.492,204.495 290.874,204.349 292.256,204.205 293.638,204.065 295.02,203.929 296.402,203.796 297.783,203.667 299.165,203.541 300.547,203.419 \n",
       "  301.929,203.301 303.311,203.186 304.693,203.075 306.075,202.968 307.457,202.865 308.839,202.765 310.22,202.67 311.602,202.578 312.984,202.49 314.366,202.406 \n",
       "  315.748,202.326 317.13,202.25 318.512,202.178 319.894,202.11 321.276,202.045 322.657,201.985 324.039,201.928 325.421,201.875 326.803,201.826 328.185,201.781 \n",
       "  329.567,201.739 330.949,201.702 332.331,201.668 333.713,201.637 335.094,201.611 336.476,201.588 337.858,201.568 339.24,201.552 340.622,201.54 342.004,201.531 \n",
       "  343.386,201.525 344.768,201.523 346.15,201.524 347.531,201.528 348.913,201.535 350.295,201.545 351.677,201.559 353.059,201.575 354.441,201.594 355.823,201.616 \n",
       "  357.205,201.64 358.587,201.668 359.969,201.698 361.35,201.73 362.732,201.765 364.114,201.802 365.496,201.842 366.878,201.883 368.26,201.927 369.642,201.973 \n",
       "  371.024,202.021 372.406,202.071 373.787,202.122 375.169,202.176 376.551,202.231 377.933,202.287 379.315,202.345 380.697,202.405 382.079,202.465 383.461,202.527 \n",
       "  384.843,202.591 386.224,202.655 387.606,202.72 388.988,202.787 390.37,202.854 391.752,202.922 393.134,202.991 394.516,203.06 395.898,203.13 397.28,203.2 \n",
       "  398.661,203.271 400.043,203.342 401.425,203.414 402.807,203.486 404.189,203.558 405.571,203.63 406.953,203.702 408.335,203.774 409.717,203.846 411.098,203.918 \n",
       "  412.48,203.989 413.862,204.061 415.244,204.132 416.626,204.203 418.008,204.273 419.39,204.343 420.772,204.412 422.154,204.481 423.535,204.549 424.917,204.617 \n",
       "  426.299,204.684 427.681,204.75 429.063,204.816 430.445,204.88 431.827,204.944 433.209,205.007 434.591,205.069 435.972,205.131 437.354,205.191 438.736,205.25 \n",
       "  440.118,205.308 441.5,205.365 442.882,205.422 444.264,205.477 445.646,205.531 447.028,205.584 448.409,205.635 449.791,205.686 451.173,205.735 452.555,205.783 \n",
       "  453.937,205.831 455.319,205.876 456.701,205.921 458.083,205.964 459.465,206.006 460.846,206.047 462.228,206.087 463.61,206.125 464.992,206.162 466.374,206.198 \n",
       "  467.756,206.232 469.138,206.265 470.52,206.297 471.902,206.328 473.283,206.357 474.665,206.385 476.047,206.412 477.429,206.438 478.811,206.462 480.193,206.485 \n",
       "  481.575,206.506 482.957,206.526 484.339,206.545 485.72,206.563 487.102,206.58 488.484,206.595 489.866,206.609 491.248,206.622 492.63,206.633 494.012,206.643 \n",
       "  495.394,206.652 496.776,206.66 498.157,206.667 499.539,206.672 500.921,206.676 502.303,206.679 503.685,206.681 505.067,206.682 506.449,206.682 507.831,206.68 \n",
       "  509.213,206.677 510.594,206.673 511.976,206.668 513.358,206.662 514.74,206.655 516.122,206.647 517.504,206.637 518.886,206.627 520.268,206.615 521.65,206.603 \n",
       "  523.031,206.589 524.413,206.575 525.795,206.559 527.177,206.543 528.559,206.525 529.941,206.507 531.323,206.487 532.705,206.467 534.087,206.445 535.469,206.423 \n",
       "  536.85,206.4 538.232,206.376 539.614,206.351 540.996,206.325 542.378,206.298 543.76,206.271 545.142,206.242 546.524,206.213 547.906,206.183 549.287,206.152 \n",
       "  550.669,206.12 552.051,206.088 553.433,206.054 554.815,206.02 556.197,205.985 557.579,205.95 558.961,205.913 560.343,205.876 561.724,205.839 563.106,205.8 \n",
       "  564.488,205.761 565.87,205.721 567.252,205.681 568.634,205.64 570.016,205.598 571.398,205.555 572.78,205.512 574.161,205.468 575.543,205.424 576.925,205.379 \n",
       "  578.307,205.334 579.689,205.287 581.071,205.241 582.453,205.194 583.835,205.146 585.217,205.097 586.598,205.048 587.98,204.999 589.362,204.949 590.744,204.898 \n",
       "  592.126,204.847 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#3da44d; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,7.87402 40.752,9.32315 42.1339,10.7747 43.5157,12.2286 44.8976,13.6851 46.2795,15.144 47.6614,16.6056 49.0433,18.0698 50.4252,19.5367 51.8071,21.0064 \n",
       "  53.189,22.4788 54.5709,23.9541 55.9528,25.4323 57.3346,26.9134 58.7165,28.3975 60.0984,29.8846 61.4803,31.3748 62.8622,32.8681 64.2441,34.3646 65.626,35.8643 \n",
       "  67.0079,37.3672 68.3898,38.8734 69.7717,40.383 71.1535,41.8958 72.5354,43.4121 73.9173,44.9318 75.2992,46.455 76.6811,47.9816 78.063,49.5118 79.4449,51.0455 \n",
       "  80.8268,52.5828 82.2087,54.1237 83.5906,55.6681 84.9724,57.2163 86.3543,58.768 87.7362,60.3234 89.1181,61.8825 90.5,63.4453 91.8819,65.0117 93.2638,66.5818 \n",
       "  94.6457,68.1556 96.0276,69.7331 97.4094,71.3142 98.7913,72.899 100.173,74.4874 101.555,76.0795 102.937,77.6751 104.319,79.2743 105.701,80.8771 107.083,82.4833 \n",
       "  108.465,84.0931 109.846,85.7062 111.228,87.3227 112.61,88.9426 113.992,90.5657 115.374,92.192 116.756,93.8214 118.138,95.4539 119.52,97.0893 120.902,98.7276 \n",
       "  122.283,100.369 123.665,102.013 125.047,103.659 126.429,105.308 127.811,106.959 129.193,108.613 130.575,110.268 131.957,111.926 133.339,113.585 134.72,115.246 \n",
       "  136.102,116.909 137.484,118.572 138.866,120.238 140.248,121.904 141.63,123.57 143.012,125.238 144.394,126.906 145.776,128.574 147.157,130.242 148.539,131.91 \n",
       "  149.921,133.577 151.303,135.243 152.685,136.909 154.067,138.573 155.449,140.236 156.831,141.897 158.213,143.556 159.594,145.212 160.976,146.866 162.358,148.516 \n",
       "  163.74,150.164 165.122,151.807 166.504,153.447 167.886,155.082 169.268,156.713 170.65,158.339 172.031,159.959 173.413,161.573 174.795,163.181 176.177,164.783 \n",
       "  177.559,166.377 178.941,167.965 180.323,169.544 181.705,171.116 183.087,172.678 184.469,174.232 185.85,175.776 187.232,177.31 188.614,178.834 189.996,180.348 \n",
       "  191.378,181.85 192.76,183.34 194.142,184.818 195.524,186.284 196.906,187.737 198.287,189.176 199.669,190.602 201.051,192.013 202.433,193.409 203.815,194.79 \n",
       "  205.197,196.156 206.579,197.505 207.961,198.838 209.343,200.154 210.724,201.452 212.106,202.733 213.488,203.995 214.87,205.239 216.252,206.464 217.634,207.67 \n",
       "  219.016,208.856 220.398,210.022 221.78,211.168 223.161,212.293 224.543,213.396 225.925,214.479 227.307,215.54 228.689,216.579 230.071,217.596 231.453,218.59 \n",
       "  232.835,219.562 234.217,220.512 235.598,221.438 236.98,222.341 238.362,223.221 239.744,224.078 241.126,224.911 242.508,225.721 243.89,226.508 245.272,227.271 \n",
       "  246.654,228.01 248.035,228.726 249.417,229.419 250.799,230.088 252.181,230.734 253.563,231.357 254.945,231.956 256.327,232.534 257.709,233.088 259.091,233.62 \n",
       "  260.472,234.13 261.854,234.619 263.236,235.085 264.618,235.531 266,235.955 267.382,236.359 268.764,236.743 270.146,237.108 271.528,237.452 272.909,237.778 \n",
       "  274.291,238.086 275.673,238.376 277.055,238.648 278.437,238.903 279.819,239.142 281.201,239.364 282.583,239.572 283.965,239.764 285.346,239.942 286.728,240.107 \n",
       "  288.11,240.258 289.492,240.396 290.874,240.523 292.256,240.638 293.638,240.742 295.02,240.836 296.402,240.919 297.783,240.994 299.165,241.06 300.547,241.118 \n",
       "  301.929,241.168 303.311,241.211 304.693,241.248 306.075,241.278 307.457,241.303 308.839,241.323 310.22,241.338 311.602,241.35 312.984,241.357 314.366,241.361 \n",
       "  315.748,241.363 317.13,241.361 318.512,241.358 319.894,241.353 321.276,241.346 322.657,241.339 324.039,241.33 325.421,241.321 326.803,241.311 328.185,241.302 \n",
       "  329.567,241.292 330.949,241.283 332.331,241.274 333.713,241.265 335.094,241.258 336.476,241.251 337.858,241.245 339.24,241.24 340.622,241.236 342.004,241.233 \n",
       "  343.386,241.231 344.768,241.23 346.15,241.231 347.531,241.232 348.913,241.234 350.295,241.238 351.677,241.242 353.059,241.247 354.441,241.253 355.823,241.259 \n",
       "  357.205,241.266 358.587,241.274 359.969,241.282 361.35,241.29 362.732,241.298 364.114,241.306 365.496,241.315 366.878,241.322 368.26,241.33 369.642,241.337 \n",
       "  371.024,241.344 372.406,241.349 373.787,241.354 375.169,241.358 376.551,241.361 377.933,241.362 379.315,241.363 380.697,241.361 382.079,241.359 383.461,241.354 \n",
       "  384.843,241.348 386.224,241.34 387.606,241.331 388.988,241.319 390.37,241.306 391.752,241.29 393.134,241.272 394.516,241.252 395.898,241.23 397.28,241.206 \n",
       "  398.661,241.18 400.043,241.151 401.425,241.12 402.807,241.087 404.189,241.052 405.571,241.014 406.953,240.975 408.335,240.933 409.717,240.889 411.098,240.843 \n",
       "  412.48,240.795 413.862,240.745 415.244,240.694 416.626,240.64 418.008,240.585 419.39,240.528 420.772,240.469 422.154,240.409 423.535,240.348 424.917,240.285 \n",
       "  426.299,240.221 427.681,240.155 429.063,240.089 430.445,240.022 431.827,239.954 433.209,239.885 434.591,239.815 435.972,239.745 437.354,239.675 438.736,239.604 \n",
       "  440.118,239.533 441.5,239.462 442.882,239.391 444.264,239.32 445.646,239.249 447.028,239.178 448.409,239.108 449.791,239.038 451.173,238.969 452.555,238.9 \n",
       "  453.937,238.832 455.319,238.765 456.701,238.699 458.083,238.634 459.465,238.571 460.846,238.508 462.228,238.446 463.61,238.386 464.992,238.328 466.374,238.27 \n",
       "  467.756,238.215 469.138,238.161 470.52,238.108 471.902,238.058 473.283,238.009 474.665,237.961 476.047,237.916 477.429,237.873 478.811,237.831 480.193,237.792 \n",
       "  481.575,237.754 482.957,237.719 484.339,237.686 485.72,237.654 487.102,237.625 488.484,237.598 489.866,237.573 491.248,237.55 492.63,237.53 494.012,237.511 \n",
       "  495.394,237.495 496.776,237.481 498.157,237.469 499.539,237.459 500.921,237.452 502.303,237.446 503.685,237.443 505.067,237.442 506.449,237.443 507.831,237.446 \n",
       "  509.213,237.451 510.594,237.458 511.976,237.467 513.358,237.478 514.74,237.491 516.122,237.506 517.504,237.523 518.886,237.541 520.268,237.562 521.65,237.584 \n",
       "  523.031,237.608 524.413,237.634 525.795,237.661 527.177,237.691 528.559,237.721 529.941,237.754 531.323,237.787 532.705,237.823 534.087,237.859 535.469,237.897 \n",
       "  536.85,237.937 538.232,237.978 539.614,238.02 540.996,238.063 542.378,238.107 543.76,238.152 545.142,238.199 546.524,238.246 547.906,238.295 549.287,238.344 \n",
       "  550.669,238.394 552.051,238.445 553.433,238.497 554.815,238.549 556.197,238.602 557.579,238.656 558.961,238.71 560.343,238.765 561.724,238.82 563.106,238.876 \n",
       "  564.488,238.932 565.87,238.988 567.252,239.045 568.634,239.102 570.016,239.159 571.398,239.216 572.78,239.273 574.161,239.33 575.543,239.388 576.925,239.445 \n",
       "  578.307,239.502 579.689,239.559 581.071,239.615 582.453,239.672 583.835,239.728 585.217,239.784 586.598,239.839 587.98,239.894 589.362,239.949 590.744,240.003 \n",
       "  592.126,240.056 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#c271d2; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,167.026 40.752,167.092 42.1339,167.16 43.5157,167.229 44.8976,167.299 46.2795,167.371 47.6614,167.444 49.0433,167.518 50.4252,167.594 51.8071,167.671 \n",
       "  53.189,167.75 54.5709,167.83 55.9528,167.912 57.3346,167.995 58.7165,168.08 60.0984,168.166 61.4803,168.254 62.8622,168.344 64.2441,168.435 65.626,168.528 \n",
       "  67.0079,168.623 68.3898,168.72 69.7717,168.818 71.1535,168.919 72.5354,169.021 73.9173,169.125 75.2992,169.231 76.6811,169.34 78.063,169.45 79.4449,169.562 \n",
       "  80.8268,169.677 82.2087,169.794 83.5906,169.913 84.9724,170.034 86.3543,170.157 87.7362,170.283 89.1181,170.411 90.5,170.542 91.8819,170.675 93.2638,170.81 \n",
       "  94.6457,170.949 96.0276,171.089 97.4094,171.233 98.7913,171.379 100.173,171.527 101.555,171.679 102.937,171.833 104.319,171.99 105.701,172.15 107.083,172.313 \n",
       "  108.465,172.479 109.846,172.648 111.228,172.82 112.61,172.995 113.992,173.173 115.374,173.354 116.756,173.538 118.138,173.726 119.52,173.917 120.902,174.111 \n",
       "  122.283,174.308 123.665,174.509 125.047,174.713 126.429,174.921 127.811,175.132 129.193,175.346 130.575,175.564 131.957,175.785 133.339,176.01 134.72,176.238 \n",
       "  136.102,176.469 137.484,176.704 138.866,176.943 140.248,177.185 141.63,177.431 143.012,177.68 144.394,177.932 145.776,178.188 147.157,178.447 148.539,178.71 \n",
       "  149.921,178.976 151.303,179.245 152.685,179.518 154.067,179.793 155.449,180.072 156.831,180.354 158.213,180.639 159.594,180.927 160.976,181.218 162.358,181.512 \n",
       "  163.74,181.808 165.122,182.107 166.504,182.409 167.886,182.713 169.268,183.019 170.65,183.328 172.031,183.638 173.413,183.951 174.795,184.265 176.177,184.582 \n",
       "  177.559,184.899 178.941,185.219 180.323,185.539 181.705,185.861 183.087,186.183 184.469,186.506 185.85,186.83 187.232,187.155 188.614,187.479 189.996,187.804 \n",
       "  191.378,188.129 192.76,188.453 194.142,188.777 195.524,189.101 196.906,189.423 198.287,189.745 199.669,190.065 201.051,190.384 202.433,190.701 203.815,191.016 \n",
       "  205.197,191.33 206.579,191.641 207.961,191.95 209.343,192.257 210.724,192.56 212.106,192.861 213.488,193.159 214.87,193.453 216.252,193.744 217.634,194.031 \n",
       "  219.016,194.315 220.398,194.595 221.78,194.87 223.161,195.141 224.543,195.408 225.925,195.671 227.307,195.929 228.689,196.182 230.071,196.43 231.453,196.673 \n",
       "  232.835,196.911 234.217,197.144 235.598,197.372 236.98,197.594 238.362,197.811 239.744,198.023 241.126,198.229 242.508,198.429 243.89,198.624 245.272,198.813 \n",
       "  246.654,198.996 248.035,199.174 249.417,199.346 250.799,199.512 252.181,199.673 253.563,199.828 254.945,199.977 256.327,200.121 257.709,200.259 259.091,200.392 \n",
       "  260.472,200.519 261.854,200.641 263.236,200.758 264.618,200.869 266,200.975 267.382,201.076 268.764,201.172 270.146,201.263 271.528,201.349 272.909,201.43 \n",
       "  274.291,201.507 275.673,201.58 277.055,201.648 278.437,201.711 279.819,201.771 281.201,201.827 282.583,201.878 283.965,201.927 285.346,201.971 286.728,202.012 \n",
       "  288.11,202.05 289.492,202.085 290.874,202.116 292.256,202.145 293.638,202.171 295.02,202.194 296.402,202.215 297.783,202.234 299.165,202.251 300.547,202.265 \n",
       "  301.929,202.278 303.311,202.288 304.693,202.297 306.075,202.305 307.457,202.311 308.839,202.316 310.22,202.32 311.602,202.323 312.984,202.325 314.366,202.326 \n",
       "  315.748,202.326 317.13,202.326 318.512,202.325 319.894,202.324 321.276,202.322 322.657,202.32 324.039,202.318 325.421,202.316 326.803,202.313 328.185,202.311 \n",
       "  329.567,202.309 330.949,202.306 332.331,202.304 333.713,202.302 335.094,202.3 336.476,202.298 337.858,202.297 339.24,202.296 340.622,202.295 342.004,202.294 \n",
       "  343.386,202.293 344.768,202.293 346.15,202.293 347.531,202.294 348.913,202.294 350.295,202.295 351.677,202.296 353.059,202.297 354.441,202.299 355.823,202.3 \n",
       "  357.205,202.302 358.587,202.304 359.969,202.306 361.35,202.308 362.732,202.31 364.114,202.312 365.496,202.314 366.878,202.316 368.26,202.318 369.642,202.32 \n",
       "  371.024,202.321 372.406,202.323 373.787,202.324 375.169,202.325 376.551,202.326 377.933,202.326 379.315,202.326 380.697,202.326 382.079,202.325 383.461,202.324 \n",
       "  384.843,202.323 386.224,202.321 387.606,202.318 388.988,202.315 390.37,202.312 391.752,202.308 393.134,202.304 394.516,202.299 395.898,202.293 397.28,202.287 \n",
       "  398.661,202.28 400.043,202.273 401.425,202.266 402.807,202.257 404.189,202.248 405.571,202.239 406.953,202.229 408.335,202.219 409.717,202.208 411.098,202.196 \n",
       "  412.48,202.184 413.862,202.172 415.244,202.159 416.626,202.146 418.008,202.132 419.39,202.117 420.772,202.103 422.154,202.088 423.535,202.072 424.917,202.057 \n",
       "  426.299,202.041 427.681,202.024 429.063,202.008 430.445,201.991 431.827,201.974 433.209,201.957 434.591,201.939 435.972,201.922 437.354,201.904 438.736,201.887 \n",
       "  440.118,201.869 441.5,201.851 442.882,201.833 444.264,201.815 445.646,201.798 447.028,201.78 448.409,201.762 449.791,201.745 451.173,201.728 452.555,201.711 \n",
       "  453.937,201.694 455.319,201.677 456.701,201.66 458.083,201.644 459.465,201.628 460.846,201.613 462.228,201.597 463.61,201.582 464.992,201.568 466.374,201.553 \n",
       "  467.756,201.539 469.138,201.526 470.52,201.513 471.902,201.5 473.283,201.488 474.665,201.476 476.047,201.465 477.429,201.454 478.811,201.444 480.193,201.434 \n",
       "  481.575,201.424 482.957,201.415 484.339,201.407 485.72,201.399 487.102,201.392 488.484,201.385 489.866,201.379 491.248,201.373 492.63,201.368 494.012,201.364 \n",
       "  495.394,201.36 496.776,201.356 498.157,201.353 499.539,201.351 500.921,201.349 502.303,201.347 503.685,201.346 505.067,201.346 506.449,201.346 507.831,201.347 \n",
       "  509.213,201.348 510.594,201.35 511.976,201.352 513.358,201.355 514.74,201.358 516.122,201.362 517.504,201.366 518.886,201.371 520.268,201.376 521.65,201.382 \n",
       "  523.031,201.388 524.413,201.394 525.795,201.401 527.177,201.408 528.559,201.416 529.941,201.424 531.323,201.433 532.705,201.441 534.087,201.451 535.469,201.46 \n",
       "  536.85,201.47 538.232,201.48 539.614,201.491 540.996,201.501 542.378,201.512 543.76,201.524 545.142,201.535 546.524,201.547 547.906,201.559 549.287,201.572 \n",
       "  550.669,201.584 552.051,201.597 553.433,201.61 554.815,201.623 556.197,201.636 557.579,201.65 558.961,201.663 560.343,201.677 561.724,201.691 563.106,201.705 \n",
       "  564.488,201.719 565.87,201.733 567.252,201.747 568.634,201.761 570.016,201.775 571.398,201.79 572.78,201.804 574.161,201.818 575.543,201.832 576.925,201.847 \n",
       "  578.307,201.861 579.689,201.875 581.071,201.889 582.453,201.903 583.835,201.917 585.217,201.931 586.598,201.945 587.98,201.959 589.362,201.973 590.744,201.986 \n",
       "  592.126,202 \n",
       "  \"/>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "426.553,104.354 574.126,104.354 574.126,28.754 426.553,28.754 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  426.553,104.354 574.126,104.354 574.126,28.754 426.553,28.754 426.553,104.354 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#009af9; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  432.553,43.874 468.553,43.874 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:start;\" transform=\"rotate(0, 474.553, 48.374)\" x=\"474.553\" y=\"48.374\">:Z</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#e26f46; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  432.553,58.994 468.553,58.994 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:start;\" transform=\"rotate(0, 474.553, 63.494)\" x=\"474.553\" y=\"63.494\">:(logistic.(Z))</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#3da44d; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  432.553,74.114 468.553,74.114 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:start;\" transform=\"rotate(0, 474.553, 78.614)\" x=\"474.553\" y=\"78.614\">:(Z .* Z)</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#c271d2; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  432.553,89.234 468.553,89.234 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:start;\" transform=\"rotate(0, 474.553, 93.734)\" x=\"474.553\" y=\"93.734\">:(logistic.(Z .* Z))</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X=-2:0.01:2\n",
    "Z=(0.4X-tanh.(X).^2)\n",
    "@plot (X, [Z, logistic.(Z), Z.*Z, logistic.(Z.*Z)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[33mWARNING: \u001b[39m\u001b[22m\u001b[33mcosh{T <: Number}(x::AbstractArray{T}) is deprecated, use cosh.(x) instead.\u001b[39m\n",
      "Stacktrace:\n",
      " [1] \u001b[1mdepwarn\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::Symbol\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./deprecated.jl:70\u001b[22m\u001b[22m\n",
      " [2] \u001b[1mcosh\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::StepRangeLen{Float64,Base.TwicePrecision{Float64},Base.TwicePrecision{Float64}}\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./deprecated.jl:57\u001b[22m\u001b[22m\n",
      " [3] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./loading.jl:515\u001b[22m\u001b[22m\n",
      " [4] \u001b[1minclude_string\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::Module, ::String, ::String\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/Compat/src/Compat.jl:577\u001b[22m\u001b[22m\n",
      " [5] \u001b[1mexecute_request\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::ZMQ.Socket, ::IJulia.Msg\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/IJulia/src/execute_request.jl:154\u001b[22m\u001b[22m\n",
      " [6] \u001b[1meventloop\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m::ZMQ.Socket\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m/home/uniwa/students2/students/20361362/linux/.julia/v0.6/IJulia/src/eventloop.jl:8\u001b[22m\u001b[22m\n",
      " [7] \u001b[1m(::IJulia.##14#17)\u001b[22m\u001b[22m\u001b[1m(\u001b[22m\u001b[22m\u001b[1m)\u001b[22m\u001b[22m at \u001b[1m./task.jl:335\u001b[22m\u001b[22m\n",
      "while loading In[83], in expression starting on line 1\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" width=\"600\" height=\"400\" viewBox=\"0 0 600 400\">\n",
       "<defs>\n",
       "  <clipPath id=\"clip00\">\n",
       "    <rect x=\"0\" y=\"0\" width=\"600\" height=\"400\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "0,400 600,400 600,0 0,0 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip01\">\n",
       "    <rect x=\"120\" y=\"0\" width=\"421\" height=\"400\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "39.3701,376.378 592.126,376.378 592.126,7.87402 39.3701,7.87402 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<defs>\n",
       "  <clipPath id=\"clip02\">\n",
       "    <rect x=\"39\" y=\"7\" width=\"554\" height=\"370\"/>\n",
       "  </clipPath>\n",
       "</defs>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  39.3701,370.85 39.3701,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  177.559,370.85 177.559,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  315.748,370.85 315.748,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  453.937,370.85 453.937,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  592.126,370.85 592.126,13.4016 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  47.6614,376.378 583.835,376.378 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  47.6614,237.32 583.835,237.32 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:0.5; fill:none\" stroke-dasharray=\"1, 2\" points=\"\n",
       "  47.6614,98.2624 583.835,98.2624 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 592.126,376.378 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 39.3701,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  177.559,376.378 177.559,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  315.748,376.378 315.748,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  453.937,376.378 453.937,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  592.126,376.378 592.126,370.85 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 39.3701,7.87402 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,376.378 47.6614,376.378 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,237.32 47.6614,237.32 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,98.2624 47.6614,98.2624 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 39.3701, 390.178)\" x=\"39.3701\" y=\"390.178\">-2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 177.559, 390.178)\" x=\"177.559\" y=\"390.178\">-1</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 315.748, 390.178)\" x=\"315.748\" y=\"390.178\">0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 453.937, 390.178)\" x=\"453.937\" y=\"390.178\">1</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:middle;\" transform=\"rotate(0, 592.126, 390.178)\" x=\"592.126\" y=\"390.178\">2</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 33.3701, 380.878)\" x=\"33.3701\" y=\"380.878\">0.0</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 33.3701, 241.82)\" x=\"33.3701\" y=\"241.82\">0.5</text>\n",
       "</g>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:end;\" transform=\"rotate(0, 33.3701, 102.762)\" x=\"33.3701\" y=\"102.762\">1.0</text>\n",
       "</g>\n",
       "<polyline clip-path=\"url(#clip02)\" style=\"stroke:#009af9; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  39.3701,7.87402 40.752,10.5541 42.1339,13.2323 43.5157,15.9083 44.8976,18.5823 46.2795,21.2542 47.6614,23.9239 49.0433,26.5914 50.4252,29.2567 51.8071,31.9196 \n",
       "  53.189,34.5803 54.5709,37.2385 55.9528,39.8943 57.3346,42.5477 58.7165,45.1985 60.0984,47.8468 61.4803,50.4924 62.8622,53.1354 64.2441,55.7756 65.626,58.4131 \n",
       "  67.0079,61.0477 68.3898,63.6795 69.7717,66.3083 71.1535,68.9342 72.5354,71.557 73.9173,74.1766 75.2992,76.7932 76.6811,79.4065 78.063,82.0165 79.4449,84.6231 \n",
       "  80.8268,87.2264 82.2087,89.8261 83.5906,92.4224 84.9724,95.015 86.3543,97.6039 87.7362,100.189 89.1181,102.77 90.5,105.348 91.8819,107.922 93.2638,110.491 \n",
       "  94.6457,113.056 96.0276,115.618 97.4094,118.175 98.7913,120.727 100.173,123.275 101.555,125.819 102.937,128.358 104.319,130.892 105.701,133.422 107.083,135.947 \n",
       "  108.465,138.467 109.846,140.982 111.228,143.491 112.61,145.996 113.992,148.495 115.374,150.989 116.756,153.477 118.138,155.96 119.52,158.437 120.902,160.908 \n",
       "  122.283,163.373 123.665,165.832 125.047,168.286 126.429,170.733 127.811,173.173 129.193,175.607 130.575,178.035 131.957,180.456 133.339,182.87 134.72,185.277 \n",
       "  136.102,187.677 137.484,190.07 138.866,192.456 140.248,194.834 141.63,197.205 143.012,199.568 144.394,201.923 145.776,204.271 147.157,206.61 148.539,208.941 \n",
       "  149.921,211.264 151.303,213.578 152.685,215.884 154.067,218.181 155.449,220.468 156.831,222.747 158.213,225.017 159.594,227.277 160.976,229.528 162.358,231.769 \n",
       "  163.74,234.001 165.122,236.222 166.504,238.433 167.886,240.634 169.268,242.824 170.65,245.004 172.031,247.173 173.413,249.331 174.795,251.477 176.177,253.613 \n",
       "  177.559,255.737 178.941,257.849 180.323,259.949 181.705,262.038 183.087,264.114 184.469,266.178 185.85,268.229 187.232,270.267 188.614,272.292 189.996,274.305 \n",
       "  191.378,276.303 192.76,278.289 194.142,280.26 195.524,282.218 196.906,284.162 198.287,286.091 199.669,288.005 201.051,289.905 202.433,291.79 203.815,293.66 \n",
       "  205.197,295.515 206.579,297.354 207.961,299.177 209.343,300.984 210.724,302.775 212.106,304.55 213.488,306.308 214.87,308.05 216.252,309.774 217.634,311.481 \n",
       "  219.016,313.171 220.398,314.843 221.78,316.497 223.161,318.133 224.543,319.75 225.925,321.35 227.307,322.93 228.689,324.492 230.071,326.034 231.453,327.557 \n",
       "  232.835,329.061 234.217,330.544 235.598,332.008 236.98,333.452 238.362,334.875 239.744,336.277 241.126,337.659 242.508,339.019 243.89,340.358 245.272,341.676 \n",
       "  246.654,342.972 248.035,344.246 249.417,345.499 250.799,346.729 252.181,347.936 253.563,349.121 254.945,350.283 256.327,351.422 257.709,352.537 259.091,353.629 \n",
       "  260.472,354.698 261.854,355.743 263.236,356.763 264.618,357.76 266,358.732 267.382,359.68 268.764,360.603 270.146,361.502 271.528,362.375 272.909,363.223 \n",
       "  274.291,364.046 275.673,364.844 277.055,365.615 278.437,366.361 279.819,367.082 281.201,367.776 282.583,368.444 283.965,369.086 285.346,369.701 286.728,370.29 \n",
       "  288.11,370.852 289.492,371.388 290.874,371.897 292.256,372.378 293.638,372.833 295.02,373.261 296.402,373.661 297.783,374.034 299.165,374.38 300.547,374.699 \n",
       "  301.929,374.99 303.311,375.253 304.693,375.489 306.075,375.697 307.457,375.878 308.839,376.03 310.22,376.156 311.602,376.253 312.984,376.322 314.366,376.364 \n",
       "  315.748,376.378 317.13,376.364 318.512,376.322 319.894,376.253 321.276,376.156 322.657,376.03 324.039,375.878 325.421,375.697 326.803,375.489 328.185,375.253 \n",
       "  329.567,374.99 330.949,374.699 332.331,374.38 333.713,374.034 335.094,373.661 336.476,373.261 337.858,372.833 339.24,372.378 340.622,371.897 342.004,371.388 \n",
       "  343.386,370.852 344.768,370.29 346.15,369.701 347.531,369.086 348.913,368.444 350.295,367.776 351.677,367.082 353.059,366.361 354.441,365.615 355.823,364.844 \n",
       "  357.205,364.046 358.587,363.223 359.969,362.375 361.35,361.502 362.732,360.603 364.114,359.68 365.496,358.732 366.878,357.76 368.26,356.763 369.642,355.743 \n",
       "  371.024,354.698 372.406,353.629 373.787,352.537 375.169,351.422 376.551,350.283 377.933,349.121 379.315,347.936 380.697,346.729 382.079,345.499 383.461,344.246 \n",
       "  384.843,342.972 386.224,341.676 387.606,340.358 388.988,339.019 390.37,337.659 391.752,336.277 393.134,334.875 394.516,333.452 395.898,332.008 397.28,330.544 \n",
       "  398.661,329.061 400.043,327.557 401.425,326.034 402.807,324.492 404.189,322.93 405.571,321.35 406.953,319.75 408.335,318.133 409.717,316.497 411.098,314.843 \n",
       "  412.48,313.171 413.862,311.481 415.244,309.774 416.626,308.05 418.008,306.308 419.39,304.55 420.772,302.775 422.154,300.984 423.535,299.177 424.917,297.354 \n",
       "  426.299,295.515 427.681,293.66 429.063,291.79 430.445,289.905 431.827,288.005 433.209,286.091 434.591,284.162 435.972,282.218 437.354,280.26 438.736,278.289 \n",
       "  440.118,276.303 441.5,274.305 442.882,272.292 444.264,270.267 445.646,268.229 447.028,266.178 448.409,264.114 449.791,262.038 451.173,259.949 452.555,257.849 \n",
       "  453.937,255.737 455.319,253.613 456.701,251.477 458.083,249.331 459.465,247.173 460.846,245.004 462.228,242.824 463.61,240.634 464.992,238.433 466.374,236.222 \n",
       "  467.756,234.001 469.138,231.769 470.52,229.528 471.902,227.277 473.283,225.017 474.665,222.747 476.047,220.468 477.429,218.181 478.811,215.884 480.193,213.578 \n",
       "  481.575,211.264 482.957,208.941 484.339,206.61 485.72,204.271 487.102,201.923 488.484,199.568 489.866,197.205 491.248,194.834 492.63,192.456 494.012,190.07 \n",
       "  495.394,187.677 496.776,185.277 498.157,182.87 499.539,180.456 500.921,178.035 502.303,175.607 503.685,173.173 505.067,170.733 506.449,168.286 507.831,165.832 \n",
       "  509.213,163.373 510.594,160.908 511.976,158.437 513.358,155.96 514.74,153.477 516.122,150.989 517.504,148.495 518.886,145.996 520.268,143.491 521.65,140.982 \n",
       "  523.031,138.467 524.413,135.947 525.795,133.422 527.177,130.892 528.559,128.358 529.941,125.819 531.323,123.275 532.705,120.727 534.087,118.175 535.469,115.618 \n",
       "  536.85,113.056 538.232,110.491 539.614,107.922 540.996,105.348 542.378,102.77 543.76,100.189 545.142,97.6039 546.524,95.015 547.906,92.4224 549.287,89.8261 \n",
       "  550.669,87.2264 552.051,84.6231 553.433,82.0165 554.815,79.4065 556.197,76.7932 557.579,74.1766 558.961,71.557 560.343,68.9342 561.724,66.3083 563.106,63.6795 \n",
       "  564.488,61.0477 565.87,58.4131 567.252,55.7756 568.634,53.1354 570.016,50.4924 571.398,47.8468 572.78,45.1985 574.161,42.5477 575.543,39.8943 576.925,37.2385 \n",
       "  578.307,34.5803 579.689,31.9196 581.071,29.2567 582.453,26.5914 583.835,23.9239 585.217,21.2542 586.598,18.5823 587.98,15.9083 589.362,13.2323 590.744,10.5541 \n",
       "  592.126,7.87402 \n",
       "  \"/>\n",
       "<polygon clip-path=\"url(#clip00)\" points=\"\n",
       "501.61,58.994 574.126,58.994 574.126,28.754 501.61,28.754 \n",
       "  \" fill=\"#ffffff\" fill-opacity=\"1\"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#000000; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  501.61,58.994 574.126,58.994 574.126,28.754 501.61,28.754 501.61,58.994 \n",
       "  \"/>\n",
       "<polyline clip-path=\"url(#clip00)\" style=\"stroke:#009af9; stroke-width:1; stroke-opacity:1; fill:none\" points=\"\n",
       "  507.61,43.874 543.61,43.874 \n",
       "  \"/>\n",
       "<g clip-path=\"url(#clip00)\">\n",
       "<text style=\"fill:#000000; fill-opacity:1; font-family:Arial,Helvetica Neue,Helvetica,sans-serif; font-size:12; text-anchor:start;\" transform=\"rotate(0, 549.61, 48.374)\" x=\"549.61\" y=\"48.374\">y1</text>\n",
       "</g>\n",
       "</svg>\n"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot(X,log(cosh(X)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "401-element Array{Float64,1}:\n",
       " -1.72935 \n",
       " -1.72397 \n",
       " -1.71857 \n",
       " -1.71315 \n",
       " -1.70769 \n",
       " -1.70221 \n",
       " -1.69671 \n",
       " -1.69117 \n",
       " -1.68561 \n",
       " -1.68001 \n",
       " -1.67439 \n",
       " -1.66874 \n",
       " -1.66306 \n",
       "  ⋮       \n",
       " -0.156738\n",
       " -0.15439 \n",
       " -0.152013\n",
       " -0.149606\n",
       " -0.14717 \n",
       " -0.144706\n",
       " -0.142213\n",
       " -0.139694\n",
       " -0.137147\n",
       " -0.134574\n",
       " -0.131974\n",
       " -0.129349"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "aforce(f,t) = (tanh(t)^2)*f(t)\n",
    "\n",
    "plot(t->aforce(logistic, t))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function Distributions.cdf(est::NeuralDensityEstimator, t::Real)\n",
    "    gr = est.sess.graph\n",
    "    ts = reshape([t], (1,1))\n",
    "    run(est.sess, est.cdf, Dict(est.t=>ts))[1]\n",
    "end\n",
    "\n",
    "function Distributions.pdf(est::NeuralDensityEstimator, t::Real)\n",
    "    gr = est.sess.graph\n",
    "    ts = reshape([t], (1,1))\n",
    "    run(est.sess, est.pdf, Dict(est.t=>ts))[1]\n",
    "end\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Returns 3 vectors.\n",
    "A vector of start points \\$t1\\$\n",
    "A vector of end points \\$t2\\$\n",
    "A vector of the counts of elements \\$ti\\$ between them, such that \\$\\left{t_i \\mid t1 \\le ti < t2 \\right}\\$\n",
    "\"\"\"\n",
    "function get_cdf_training_pairs(observations)\n",
    "    observations = sort(observations)\n",
    "    \n",
    "    t1s = Float32[]\n",
    "    t2s = Float32[]\n",
    "    probs = Float32[]\n",
    "    for (ii, t1) in enumerate(observations)\n",
    "        for jj in ii+1 : length(observations)\n",
    "            t2 = observations[jj]\n",
    "            count = jj - ii           \n",
    "            push!(t1s, t1)\n",
    "            push!(t2s, t2)\n",
    "            push!(probs, count/length(observations))\n",
    "        end\n",
    "    end\n",
    "    \n",
    "    t1s, t2s, probs\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "function StatsBase.fit!(estimator::NeuralDensityEstimator, observations;\n",
    "    epochs = 20,\n",
    "    batch_size = 1024)\n",
    "    gr = estimator.sess.graph\n",
    "    \n",
    "    for ii in 1:epochs\n",
    "        batch_losses = Float32[]\n",
    "        for (t1_o, t2_o, probs_o) = eachbatch(shuffleobs(get_cdf_training_pairs(observations)), batch_size)\n",
    "            loss_o, _, = run(estimator.sess, \n",
    "                [gr[\"loss\"], estimator.optimizer],\n",
    "                Dict(gr[\"t1\"]=>t1_o', gr[\"t2\"]=>t2_o', gr[\"countprob_between\"]=>probs_o))\n",
    "            push!(batch_losses, loss_o[1])\n",
    "        end\n",
    "        epoch_loss = mean(batch_losses)\n",
    "        println(\"Epoch $ii: loss: $(epoch_loss)\")\n",
    "    end\n",
    "    estimator\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "est = NeuralDensityEstimator([64, 64])\n",
    "data = GenerateDatasets.magdon_ismail_and_atiya();\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fit!(est, data; epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "histogram(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X=-50:0.1:100\n",
    "\n",
    "plot(X, pdf.(est, X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "empirical_cdf(data, X) = [length(filter(i->i<x, data)) for x in X]./length(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot(X, [cdf.(est, X), empirical_cdf(data, X)] , label=[\"Estimated\" \"Empirical\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "2π*sqrt(0.296/9.8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "2π*sqrt(0.296/11.7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 0.6.0",
   "language": "julia",
   "name": "julia-0.6"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "0.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
